package com.john.study.es.fetcher;

import com.john.common.util.codec.Md5Util;
import com.john.common.util.http.HttpClientInitException;
import com.john.common.util.http.HttpRequestUtils;
import com.john.common.util.http.HttpResponseStatusException;
import com.john.study.es.module.ChinaUnicomAllPageVo;
import org.apache.commons.lang.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.URISyntaxException;
import java.util.Iterator;

/**
 * @author jiangguangtao 2017/7/28.
 */
public class WebFetcher {
    private static final Logger LOGGER = LoggerFactory.getLogger(WebFetcher.class);

    public ChinaUnicomAllPageVo fetcher(FetchContext context) {
        String url = context.getUrl();

        String content = null;
        try {
            content = HttpRequestUtils.get(url);
        } catch (URISyntaxException | HttpResponseStatusException
                | HttpClientInitException | IOException e) {
            LOGGER.error("请求Web异常 {}", url, e);
            return null;
        }
        ChinaUnicomAllPageVo pageVo = new ChinaUnicomAllPageVo();

        Document document = Jsoup.parse(content, url);
        String title = document.title();
        Element keywordsEle = document.select("head>meta[name='keywords']").first();
        String keywords = null == keywordsEle ? null : keywordsEle.attr("content");
        Element descEle = document.select("head>meta[name='description']").first();
        String desc = null == descEle ? null : descEle.attr("content");
//        LOGGER.info("标题：{}", title);
//        LOGGER.info("关键字：{}", keywords);
//        LOGGER.info("描述：{}", desc);
        pageVo.setUrl(url);
        pageVo.setTitle(title);
        pageVo.setKeywords(keywords);
        pageVo.setDescription(desc);

        Element body = document.select("body").first();
//        LOGGER.info("{}", body.text());
        pageVo.setContent(body.text());
        pageVo.setPageKey(Md5Util.stringToMD5(pageVo.getContent()));
        pageVo.setUrlKey(Md5Util.stringToMD5(pageVo.getUrl()));

        if (context.isFetchUrls()) {
            Elements links = document.select("body a[href]");
            Iterator<Element> iterator = links.iterator();
            LOGGER.trace("当前页面上的地址列表：");
            int i = 0;
            while (iterator.hasNext()) {
                Element link = iterator.next();
                String href = StringUtils.trimToNull(link.attr("href"));
                i++;
                context.addUrl(href);
            }
            LOGGER.info("单面链接数：{}", i);
        }
        return pageVo;
    }

}
